package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
)

//NOTICE:  该例子存在问题
type Response struct {
	Url  string
	Body string
	Meta *map[string]string
}

type Request struct {
	Url   string
	Parse func(response *Response)
	Meta  *map[string]string
}

func ExampleParse(response *Response) {
	// TODO
}

var RequestChan = make(chan *Request, 100)   // 请求池
var DataChan = make(chan *map[string]string) // 数据池

var wg sync.WaitGroup //定义一个同步等待的组
//var swg sync.WaitGroup // 等待数据处理结束

var downloadProcess = func(url string) (content string, statusCode int) {
	defer func() {
		fmt.Printf("get<%d>: %s\n", statusCode, url)
	}()
	resp, err := http.Get(url)
	if err != nil {
		statusCode = -100
		return
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		statusCode = -200
		return
	}
	statusCode = resp.StatusCode
	content = string(data)
	return
}

func doRequest(request *Request) {
	defer wg.Done()
	html, sta := downloadProcess(request.Url)
	if sta != 200 {
		return
	}
	request.Parse(&Response{request.Url, html, request.Meta})
}
func saveProcess() {
	//defer swg.Done()
	for {
		data := <-DataChan
		if data == nil {
			break
		}
		fmt.Printf("====> save")
		fmt.Println(data)
	}
}

// 调度器
func Scheduler(threads int) {
	go saveProcess()
	//swg.Add(1)
	//for {
	for i := 0; i < threads; i++ {
		req := <-RequestChan
		if req == nil {
			DataChan <- nil
			return
		}
		go doRequest(req)
		wg.Add(1)
	}
	wg.Wait()
	//}
	//swg.Wait()
}

func parse(response *Response) {
	fmt.Println(response.Url)

	data := make(map[string]string)
	data["text"] = response.Body
	DataChan <- &data
}

func main() {
	RequestChan <- &Request{
		Url:   "http://ip.wlwz620.com/",
		Parse: parse,
		Meta:  nil,
	}
	Scheduler(1)
}
